# CoverageEffectiveness.R

# Description: This R program contains a collection of functions that
# are used to calculate the coverage effectiveness (CE) of a test
# suite and to identify prioritizations that maximize coverage
# effectiveness.

# Global Variables:

# This is the simple working coverage file with just a few test cases
# and variables; Note that this data file format uses the full names
# of the test cases and thus it is not really useful for either
# calculating CE or maximizing a CE function
CoverageDataFile = "CoverageData.txt"
TestTimingDataFile = "TestTimingData.txt"

# This is the slightly extended version that comes from my PhD
# dissertation (see chapter eight).  Note that the test case
# information after the third test was just recently made up
# on a whim to furhter experiment with function maximization
DissCoverageDataFile = "DissCoverageData.txt"
DissTestTimingDataFile = "DissTestTimingData.txt"

# This is the extended dissertation version that contains a bunch
# of test cases (seven total) and their requirements
DissCoverageDataFileLarge = "DissCoverageDataLarge.txt"
DissTestingTimingDataFileLarge = "DissTestTimingDataLarge.txt"

# This is the cost of running a test suite under the unit cost
# assumption (used during the calculation of CE_u)
UnitCost = 1

# This is the penalty that we assign to a test suite that does not
# contain all of the test cases (RGenoud can cause this to happen)
BadFitnessPenalty = -9999999

# These are the global variables that store the coverage and timing
# data for the test suite.  The data is stored in a global manner
# because the function maximizers can only maximize a function that
# accepts the regular input parameters (i.e., the test tuple itself).
# Using global variables is better than regularly reading in the file.
GlobalCoverage = c()
GlobalTiming = c()

# FUNCTION:
# Input the coverage data from the specified file.
ReadCoverageData = function(CoverageDataFile)
{

  # read in the coverage information
  Coverage = read.table(file=CoverageDataFile, header=T);
  return(Coverage)
  
}

# FUNCTION:
# Input the timing data from the specified file.
ReadTestTimingData = function(TimingDataFile)
{

  # read in the timing information
  Timing = read.table(file=TimingDataFile, header=T);
  return(Timing)

}

# FUNCTION:
# This function stores the global information about the coverage
# and timing of the test suite.  The global variables are used
# because the function maximizers only accept the input parameters
# to the CE function.
SetGlobals = function(GlobalReqCoverage, GlobalReqTiming)
{

  GlobalCoverage <<- GlobalReqCoverage
  GlobalTiming <<- GlobalReqTiming

}

# FUNCTION:
# A debugging function that ensures the global variables are being
# set properly.
PrintGlobals = function()
{

  #print(GlobalCoverage)
  #print(GlobalTiming)

}

# FUNCTION:
# Calculate coverage effectiveness for all permutations of a given
# ordering.  Note that this code can be used if we want to
# exhaustively explore the space of all possible prioritizations (for
# small test suites).  See the PrioritizeExhaustive function.
AllCoverageEffectiveness = function(Coverage, Timing, AllGraphs=FALSE)
{

  # initialize the list of the CE values to empty
  AllOrderingsCoverageEffectiveness = c()

  # initialize the list of the different orderings
  TotalOrderingsJustNames = c();
  
  # get all of the unique test cases and sort them so
  # that they appear in a predictable ordering
  UniqueTestCases = sort(unique(Coverage$NameCD))

  # enumerate all possible permutations of the test suite; since the
  # test suite is of size n, then we are asking for n! total number of
  # test suites (not feasible for moderate to large test suites)
  AllOrderings = permutations(length(UniqueTestCases),
    length(UniqueTestCases), UniqueTestCases);

  # calculate the number of rows inside of the permutation listing
  # (note that this is probably not needed since we know that there
  # has to be n! orderings!)
  NumberOfRows = (length(AllOrderings) %/% length(UniqueTestCases))

  # we are supposed to make all of the graphs and thus we need to
  # develop a plotting region that shows all of the graphs in a grid
  # structure (otherwise, no graphing region will be developed)
  if( AllGraphs == TRUE )
    {
  
      # make a graphics plotting region that will display all of the
      # graphs in the same X11 area (often, they are not perfectly
      # formatted, but this is a nice way to quickly compare a number
      # of different prioritizations)
      par(mfrow = c(NumberOfRows/3, 3))

    }
      
  # iterate through all of the rows inside of the data table; note that
  # each row corresponds to one of the prioritizations that was
  # generated by the permutations methods
  for ( i in 1:(NumberOfRows) )
    {

      CurrentCEValue =
        CalculateCoverageEffectiveness(AllOrderings[i,],
                                       Coverage, Timing,
                                       Graph=AllGraphs)

      TotalOrderingsJustNames =
        c(TotalOrderingsJustNames, paste(AllOrderings[i,],
                                         collapse=", "))
      
      AllOrderingsCoverageEffectiveness =
        c(AllOrderingsCoverageEffectiveness, CurrentCEValue)
      
    }

  Ordering = TotalOrderingsJustNames
  CE = AllOrderingsCoverageEffectiveness
  
  return(data.frame(Ordering, CE))

}

# FUNCTION: Calculate the coverage effectiveness for a function
# maximizer usch as SANN.  This function calls the regular CalculateCE
# function with the currently set test suite coverage and timing data.
CoverageEffectiveness = function(Ordering)
{

  # debugging information
##   print(GlobalCoverage)
##   print(GlobalTiming)

  # read in the coverage and timing data from the file system 
##   CoverageData = ReadCoverageData(DissCoverageDataFileLarge)
##   TimingData = ReadTimingData(DissTimingDataFileLarge)

  # use the coverage and timing data that was placed inside
  # of the designated global variables
  CoverageData = GlobalCoverage
  TimingData = GlobalTiming

  # use the helper function to calculate the CE value -- note
  # that since this is being used during function maximization
  # we do not need to plot the graphs
  return(CalculateCoverageEffectiveness(Ordering, CoverageData,
                                        TimingData, Graph = FALSE))

}

# FUNCTION: Calculate the coverage effectiveness for a function
# maximizer usch as the GA.  This function calls the regular
# CalculateCE function with the currently set test suite coverage and
# timing data.
CoverageEffectivenessGA = function(Ordering)
{

  # use the coverage and timing data that was placed inside
  # of the designated global variables
  CoverageData = GlobalCoverage
  TimingData = GlobalTiming

  # extract the length of the test suite from the TimingData
  TestSuite = TimingData$NameTTD
  TestSuiteSize = length(TestSuite)

  # determine the "unique length" of the ordering that was input to
  # this function; the genetic algorithm will try to come up with
  # orderings that do not use all of the test cases
  UniqueOrderingLength = length(unique(Ordering))
  
  # debugging information
##   print(TestSuite)
##   print(TestSuiteSize)
##   print(UniqueOrderingLength)

  # we have found an ordering that does not include all of the
  # test cases and so we should penalize it with a bad fitness
  if( UniqueOrderingLength < TestSuiteSize )
    {

      return(BadFitnessPenalty)

    }
  
  # use the helper function to calculate the CE value -- note
  # that since this is being used during function maximization
  # we do not need to plot the graphs
  return(CalculateCoverageEffectiveness(Ordering, CoverageData,
                                        TimingData, Graph = FALSE))

}

# FUNCTION: This function calculates a CE value and it can plot the
# coverage function for the provided ordering.  This calculation
# involves the creation of two step functions and the numerical
# approximation of the integral of each step function.
CalculateCoverageEffectiveness = function(Ordering,
  Coverage, Timing, Graph = TRUE)
{

  # attach the Coverage and Timing so that we can use the names of the
  # individual fields without having to fully qualify them
##   attach(Coverage);
##   attach(Timing);

  # the cummulative coverage of the test requirements
  # note that initially none of the requirements have
  # been covered by the test suite
  TotalCoverage = c();

  # the jump locations inside of the CE step function
  # note that the first value is always zero 
  JumpLocations = c();

  # the between heights for the CE step function
  # note that the first value is also always zero
  BetweenHeights = c(0)

  # this is the running time for "executing" the test suite
  RunningTime = 0
  
  # tried this and I ran into parameter passing problems
  #sapply(Ordering, GetUniqueCoverage)
  
  # apply the function called GetUniqueCoverage to the
  # Ordering that was provided by the input parameter
  for ( CurrentTestCase in Ordering )
    {

       # extract the requirements for the CurrentTestCase
      CurrentRequirements =
        Coverage$Requirement[Coverage$NameCD == CurrentTestCase]

      # extract the running time for the CurrentTestCase
      CurrentTime = Timing$Time[Timing$NameTTD == CurrentTestCase]
      
      # determine how many unique requirements this test
      # case actually covered (probably not needed)
      MyCoveredRequirements =
        setdiff(CurrentRequirements, TotalCoverage)

      # update the total coverage as we "run" this next test case
      TotalCoverage = union(TotalCoverage, CurrentRequirements)

      # add in the next between height
      BetweenHeights = c(BetweenHeights, length(TotalCoverage))

      # add in the next jump location
      JumpLocations = c(JumpLocations, RunningTime+CurrentTime)

      # update the running time to reflect the fact that
      # we "ran" the CurrentTestCase
      RunningTime = RunningTime + CurrentTime
  
    }

  # debugging information
##   print(c("TC", TotalCoverage))
##   print(c("JL", JumpLocations))
##   print(c("BH", BetweenHeights))
##   print(c("RT", RunningTime))

  # create the step function from the JumpLocations and BetweenHeights
  KappaStepFunction = stepfun(JumpLocations, BetweenHeights)

  # integrate the requirement coverage function from zero
  # to the time when the last test case finishes execution
  # (note that this corresponds to the running time)
  KappaStepFunctionArea = integrate(KappaStepFunction, lower=0,
    upper=RunningTime,subdivisions=1000)

  # debugging information
##   print(c("KappaArea", KappaStepFunctionArea$value))

  # define the BetweenHeights for the constant function that
  # corresponds to the ideal coverage of the test requirements --
  # that is all of the requirements are covered immediately
  ConstantFunction = function(x){  return(length(TotalCoverage)) }
  ConstantBetweenHeights = sapply(BetweenHeights, ConstantFunction)

  # debugging information
##   print(c("CBH", ConstantBetweenHeights))
  
  # create the constant step function from the modification of the
  # data already collected (but, it attains coverage immediately)
  ConstantStepFunction = stepfun(JumpLocations,
    ConstantBetweenHeights)

  # integrate the requirement coverage function from zero
  # to the time when the last test case finishes execution
  # (note that this corresponds to the running time)
  ConstantStepFunctionArea = integrate(ConstantStepFunction, lower=0,
    upper=RunningTime,subdivisions=1000)
  
  # detach the Coverage and Timing so that we do not have aliasing
  # and the functions can be called many times without problems
##   detach(Coverage);
##   detach(Timing);

  # calculate the final coverage effectiveness value
  CoverageEffectiveness =
    (KappaStepFunctionArea$value / ConstantStepFunctionArea$value)

  # the parameter specified that a graph should be produced
  if( Graph == TRUE )
    {
  
      # make the final coverage effectiveness label
      CELabel = c("CE = ", signif(CoverageEffectiveness,digits=4))
      CELabelFinal = paste(CELabel, collapse="")

      # this did not work; I could not combine the math with the value
      # of a variable; I have not yet checked online
      ##   expression(paste(xi(T,T[p]), CELabelFinal, collapse=""))
      ##   print(CELabelFinal)

      # make the final ordering label
      FirstOrdering = paste(Ordering, collapse=", ")
      OrderingLabel = c(FirstOrdering, " : ", CELabelFinal)
      OrderingLabelFinal = paste(OrderingLabel, collapse="")
##       print("OLF")
##       print(OrderingLabelFinal)
      
      # main = CELabelFinal,
      
      # plot the step function that corresponds to kappa, or the
      # requirement coverage function
      plot(KappaStepFunction,
           xlab = expression(paste("Testing Time  ", (italic(t)))),
           ylab = expression(paste("Covered Test Requirements  ",
               (C(T,t)))),
           main = OrderingLabelFinal,
           cex.main=.975,
           font.main = 1
           )

##            sub = CELabelFinal
      
##       par(xpd = TRUE);
##        par(usr=c(0,1,0,1))
##        text(x=.2, y=1.1, CELabelFinal)
      
    }

  # return the value of coverage effectiveness that was calculated
  # (this output can be used in other graphs and examined by an RUnit
  # test suite
  return(CoverageEffectiveness)
  
}

# FUNCTION: Calculate coverage effectiveness for all permutations of a
# given ordering.  Note that this code can be used if we want to
# exhaustively explore the space of all possible prioritizations (for
# small test suites).  See the PrioritizeExhaustive function.
AllUnitCoverageEffectiveness = function(Coverage, Timing, AllGraphs=FALSE)
{

  # initialize the list of the CE values to empty
  AllOrderingsCoverageEffectiveness = c()

  # initialize the list of the different orderings
  TotalOrderingsJustNames = c();
  
  # get all of the unique test cases and sort them so
  # that they appear in a predictable ordering
  UniqueTestCases = sort(unique(Coverage$NameCD))

  # enumerate all possible permutations of the test suite; since the
  # test suite is of size n, then we are asking for n! total number of
  # test suites (not feasible for moderate to large test suites)
  AllOrderings = permutations(length(UniqueTestCases),
    length(UniqueTestCases), UniqueTestCases);

  # calculate the number of rows inside of the permutation listing
  # (note that this is probably not needed since we know that there
  # has to be n! orderings!)
  NumberOfRows = (length(AllOrderings) %/% length(UniqueTestCases))

  # make a graphics plotting region that will display all of the
  # graphs in the same X11 area (often, they are not perfectly
  # formatted, but this is a nice way to quickly compare a number
  # of different prioritizations)
  par(mfrow = c(NumberOfRows/3, 3))
  
  # iterate through all of the rows inside of the data table; note that
  # each row corresponds to one of the prioritizations that was
  # generated by the permutations methods
  for ( i in 1:(NumberOfRows) )
    {

      CurrentCEValue =
        CalculateUnitCoverageEffectiveness(AllOrderings[i,],
                                           Coverage, Timing,
                                           Graph=AllGraphs)

      TotalOrderingsJustNames =
        c(TotalOrderingsJustNames, paste(AllOrderings[i,],
                                         collapse=", "))
      
      AllOrderingsCoverageEffectiveness =
        c(AllOrderingsCoverageEffectiveness, CurrentCEValue)
      
    }

  Ordering = TotalOrderingsJustNames
  CE = AllOrderingsCoverageEffectiveness
  
  return(data.frame(Ordering, CE))

}

# FUNCTION: Calculate the unit coverage effectiveness (CE_u) for a
# function maximizer such as SANN.  This function calls the regular
# CalculateUnitCE function with the currently set test suite coverage and
# timing data.
UnitCoverageEffectiveness = function(Ordering)
{

  # use the coverage and timing data that was placed inside
  # of the designated global variables
  CoverageData = GlobalCoverage
  TimingData = GlobalTiming

  # use the helper function to calculate the CE value -- note
  # that since this is being used during function maximization
  # we do not need to plot the graphs
  return(CalculateUnitCoverageEffectiveness(Ordering, CoverageData,
                                            TimingData, Graph = FALSE))

}

# FUNCTION: This function calculates a CE_u value and it can plot the
# coverage function for the provided ordering.  This calculate
# involves the creation of two step functions and the numerical
# approximation of the integral of each step function.
CalculateUnitCoverageEffectiveness = function(Ordering,
  Coverage, Timing, Graph = TRUE)
{

  # the cummulative coverage of the test requirements
  # note that initially none of the requirements have
  # been covered by the test suite
  TotalCoverage = c();

  # the jump locations inside of the CE step function
  # note that the first value is always zero 
  JumpLocations = c();

  # the between heights for the CE step function
  # note that the first value is also always zero
  BetweenHeights = c(0)

  # this is the running time for "executing" the test suite
  RunningTime = 0
  
  # apply the function called GetUniqueCoverage to the
  # Ordering that was provided by the input parameter
  for ( CurrentTestCase in Ordering )
    {

       # extract the requirements for the CurrentTestCase
      CurrentRequirements =
        Coverage$Requirement[Coverage$NameCD == CurrentTestCase]

      # UNIT COST ASSUMPTION: assume that the cost associated with
      # running all of the test cases is the same (note that this
      # is in contrast to running the regular CalcCE, where you
      # would extract this value from the data file)
      CurrentTime = UnitCost
      
      # determine how many unique requirements this test
      # case actually covered (probably not needed)
      MyCoveredRequirements =
        setdiff(CurrentRequirements, TotalCoverage)

      # update the total coverage as we "run" this next test case
      TotalCoverage = union(TotalCoverage, CurrentRequirements)

      # add in the next between height
      BetweenHeights = c(BetweenHeights, length(TotalCoverage))

      # add in the next jump location
      JumpLocations = c(JumpLocations, RunningTime+CurrentTime)

      # update the running time to reflect the fact that
      # we "ran" the CurrentTestCase
      RunningTime = RunningTime + CurrentTime
  
    }

  # create the step function from the JumpLocations and BetweenHeights
  KappaStepFunction = stepfun(JumpLocations, BetweenHeights)

  # integrate the requirement coverage function from zero
  # to the time when the last test case finishes execution
  # (note that this corresponds to the running time)
  KappaStepFunctionArea = integrate(KappaStepFunction, lower=0,
    upper=RunningTime,subdivisions=1000)

  # define the BetweenHeights for the constant function that
  # corresponds to the ideal coverage of the test requirements --
  # that is all of the requirements are covered immediately
  ConstantFunction = function(x){  return(length(TotalCoverage)) }
  ConstantBetweenHeights = sapply(BetweenHeights, ConstantFunction)

  # create the constant step function from the modification of the
  # data already collected (but, it attains coverage immediately)
  ConstantStepFunction = stepfun(JumpLocations,
    ConstantBetweenHeights)

  # integrate the requirement coverage function from zero
  # to the time when the last test case finishes execution
  # (note that this corresponds to the running time)
  ConstantStepFunctionArea = integrate(ConstantStepFunction, lower=0,
    upper=RunningTime,subdivisions=1000)
  
  # calculate the final coverage effectiveness value
  CoverageEffectiveness =
    (KappaStepFunctionArea$value / ConstantStepFunctionArea$value)

  # the parameter specified that a graph should be produced
  if( Graph == TRUE )
    {
  
      # make the final coverage effectiveness label
      CELabel = c("CE = ", signif(CoverageEffectiveness,digits=4))
      CELabelFinal = paste(CELabel, collapse="")

      # make the final ordering label
      FirstOrdering = paste(Ordering, collapse=", ")
      OrderingLabel = c(FirstOrdering, " : ", CELabelFinal)
      OrderingLabelFinal = paste(OrderingLabel, collapse="")

      # plot the step function that corresponds to kappa, or the
      # requirement coverage function
      plot(KappaStepFunction,
           xlab = expression(paste("Testing Time  ", (italic(t)))),
           ylab = expression(paste("Covered Test Requirements  ",
               (C(T,t)))),
           main = OrderingLabelFinal,
           cex.main=.975,
           font.main = 1
           )

    }

  # return the value of coverage effectiveness that was calculated
  # (this output can be used in other graphs and examined by an RUnit
  # test suite)
  return(CoverageEffectiveness)
  
}

# FUNCTION: This function constructs the step function that is the
# inverse of the traditional C function.  Refactoring and making this
# a separate function makes sense because we need to call it multiple
# times inside of the CalculateCoverageEffectivenessDelta function.
CreateCInverseFunction = function(Ordering, Coverage, Timing)
{

  # the cummulative coverage of the test requirements
  # note that initially none of the requirements have
  # been covered by the test suite
  TotalCoverage = c();

  # the jump locations inside of the CE step function
  # note that the first value is always zero 
  JumpLocations = c();

  # the between heights for the CE step function
  # note that the first value is also always zero
  BetweenHeights = c(0)

  # this is the running time for "executing" the test suite
  RunningTime = 0
  
  # apply the function called GetUniqueCoverage to the
  # Ordering that was provided by the input parameter
  for ( CurrentTestCase in Ordering )
    {

      # extract the requirements for the CurrentTestCase
      CurrentRequirements =
        Coverage$Requirement[Coverage$NameCD == CurrentTestCase]

      # extract the running time for the CurrentTestCase
      CurrentTime = Timing$Time[Timing$NameTTD == CurrentTestCase]
      
      # determine how many unique requirements this test
      # case actually covered (probably not needed)
      MyCoveredRequirements =
        setdiff(CurrentRequirements, TotalCoverage)

      # update the total coverage as we "run" this next test case
      TotalCoverage = union(TotalCoverage, CurrentRequirements)

      # add in the next between height
      # OLD:
      #BetweenHeights = c(BetweenHeights, length(TotalCoverage))

      # INVERSE:
      BetweenHeights = c(BetweenHeights, RunningTime+CurrentTime)
      
      # add in the next jump location
      # OLD:
      #JumpLocations = c(JumpLocations, RunningTime+CurrentTime)

      # INVERSE:
      JumpLocations = c(JumpLocations, length(TotalCoverage))
      
      # update the running time to reflect the fact that
      # we "ran" the CurrentTestCase
      RunningTime = RunningTime + CurrentTime
  
    }

  # debugging information
  #print(c("TC", TotalCoverage))
  #print(c("JL", JumpLocations))
  #print(c("BH", BetweenHeights))
  #print(c("RT", RunningTime))

  # create the step function from the JumpLocations and BetweenHeights
  KappaStepFunction = stepfun(JumpLocations, BetweenHeights)
  return(KappaStepFunction)
  
}

# FUNCTION: This function calculates a CE_delta value and it can plot
# the coverage function for the provided ordering.  This calculation
# involves the creation of two step functions and the numerical
# approximation of the integral of each step function.  Also note
# that this requires the timing information before and after the
# prioritization of the test suite (prioritization to remove database
# restarts can make the test suite run faster).  However, note that
# the coverage relationship will still hold after prioritization.
CalculateCoverageEffectivenessDelta = function(OrderingBefore,
  OrderingAfter, Coverage, TimingBefore, TimingAfter, Graph = TRUE)
{

  # determine the number of unique requirements that are covered
  # by the test suite (extract this from the Coverage parameter)
  UniqueRequirementsCount = length(unique(Coverage$Requirement))

  # BEFORE STEP FUNCTION :
  
  # construct the step function for the timing data before we
  # performed the restart-removal prioritization
  BeforeStepFunction =
    CreateCInverseFunction(OrderingBefore, Coverage, TimingBefore)
  
  # integrate the BeforeStepFunction to calculate the area
  BeforeStepFunctionArea = integrate(BeforeStepFunction, lower=0,
    upper=UniqueRequirementsCount,subdivisions=1000)

  # AFTER STEP FUNCTION :
  
  # construct the step function for the timing data before we
  # performed the restart-removal prioritization
  AfterStepFunction =
    CreateCInverseFunction(OrderingAfter, Coverage, TimingAfter)
  
  # integrate the AfterStepFunction to calculate the area
  AfterStepFunctionArea = integrate(AfterStepFunction, lower=0,
    upper=UniqueRequirementsCount,subdivisions=1000)

  # calculate the final coverage effectiveness delta value
  CoverageEffectivenessDelta = ((BeforeStepFunctionArea$value - AfterStepFunctionArea$value ) / BeforeStepFunctionArea$value)

  #print(CoverageEffectivenessDelta)
  
  # the parameter specified that a graph should be produced
  if( Graph == TRUE )
    {
  
      # make the final coverage effectiveness label
      CELabel = c("CED = ", signif(CoverageEffectivenessDelta,digits=4))
      CELabelFinal = expression(paste(CE[Delta], CELabel))

      # make the final ordering label (for BEFORE prioritization)
      FirstOrdering = paste(OrderingBefore, collapse=", ")
      OrderingLabel = c(FirstOrdering, " : ", CELabelFinal)
      OrderingLabelFinal = paste(OrderingLabel, collapse="")

      par(xpd=FALSE)
      
      # plot the step function that corresponds to the timing and
      # coverage before we prioritized and removed the restarts
      plot(BeforeStepFunction, 
           ylab = expression(paste("Testing Time  ", (italic(t)))),
           xlab = expression(paste("Covered Test Requirements  ",
               italic((C(T,t))))),
           cex.main=.975,
           main = "",
           font.main = 1
           )

      # plot the step function that corresponds to the timing and
      # coverage after we prioritized and removed the restarts
      plot(AfterStepFunction, 
           ylab = expression(paste("Testing Time  ", (italic(t)))),
           xlab = expression(paste("Covered Test Requirements  ",
               (italic(C(T,t))))),
           cex.main=.975,
           font.main = 1,
           main = "",
           lty="dashed",
           add=TRUE
           )

      # set up the graph so that we can place the legend outside
      # of the box (notice that we do this after plotting the curves
      # because otherwise the curves will flow out of the box)
      par(xpd=TRUE)
      par(usr=c(0,1,0,1))

      #text(adj = c(1, 1), OrderingLabelFinal)
      #text(x=.85, y=1.05, OrderingLabelFinal)
      cevalue = signif(CoverageEffectivenessDelta,digits=4)
      text(x=.85, y=1.05, bquote((CE[Delta]) == .(cevalue)))          
      
      # create the legend and then display it inside of the graph
      # note that we assume that the prioritizations are going to
      # be called T and T' and that T' is after and T is before
      finalLegend = c("T", expression(paste("T'")))
      legend(x=0, y=1.09, finalLegend,
        bty="n", horiz=TRUE, col = c("black", "black"), 
        lty=1:2)
      
    }

  # return the value of coverage effectiveness that was calculated
  # (this output can be used in other graphs and examined by an RUnit
  # test suite
  return(CoverageEffectivenessDelta)
  
}

# FUNCTION: This function calculates the ratio between the total
# number of covered test requirements and the time that is required to
# run the entire test suite.  This is a useful evaluation metric when
# you are evaluating prioritization techniques that avoid database
# restarts.  It is useful because removal of restarts reduces the
# entire time that is required to run the test suite.  Note that this
# can be calculated regardless of the ordering -- only because of
# the fact that avoiding restarts would reduce time and not change
# the overall coverage.
CalculateCoverageToTimeRatio = function(Coverage, Timing)
{

  # calculate the total number of unique requirements that
  # are covered by the test suite
  UniqueRequirementsCount = length(unique(Coverage$Requirement))

  # calculate the total execution time for the test suite
  TotalExecutionTime = sum(Timing$Time)

  # make the final calculation of the ratio and then return
  return((UniqueRequirementsCount/TotalExecutionTime))

}

# FUNCTION: This function creates the summary statistics for the CE
# values for a large number of prioritizations.  This is useful
# because you can then compare how the values vary for different
# metrics.  For instance, this can be called to compare CE to other
# metrics (e.g., APBC) for the same prioritizations.
SummarizePrioritizationCEScores = function(AllPrioritizations)
{

  # set the configuration options so that it is easier
  # to read the data output (read the documentation for
  # the pastecs library in order to learn more about the
  # options that this tool provides)
  options(scipen=100)
  options(digits=2)

  # collect all of the descriptive statistics
  describe = stat.desc(AllPrioritizations$CE)
  return(describe)
  
}

# FUNCTION: This function will plot a scatter plot of all of the CE
# values for the orderings that have been calculated.  It can be
# configured to create a black and white or a color version.
#
# CALLING EXAMPLE:
#
# GraphScatterPlot(AllCEU, Color=TRUE)
GraphScatterPlot = function(AllPrioritizations, Color = FALSE)
{

  # make the color plot (fill in the circles with blue)
  if( Color == TRUE )
    {

      X11(height=6, width=6)
      plot(AllCEU$CE, pch=21, bg="lightsteelblue", lwd=0, xlab="Test Prioritization Index", ylab = "Coverage Effectiveness Value")

    }

  # make the black and white (greyscale) plot for the purposes
  # of publication
  else
    {

      

    }
  
}

# FUNCTION: This function will save the current working graph to the
# file that is specified.  Note that the X11 device that is currently
# labelled as (ACTIVE) will be saved.
SaveGraphic = function(fileName)
{
  
  dev.copy(postscript, file=fileName, height=6, width=6,
           horizontal=F, onefile=F)
  dev.off()

}

# FUNCTION: This function will save the current working graph to the
# file that is specified.  Note that the X11 device that is currently
# labelled as (ACTIVE) will be saved.
SaveGraphicPaper = function(fileName)
{
  
  SaveGraphic(fileName)

}

# FUNCTION: This function will save the current working graph to the
# file that is specified.  Note that the X11 device that is currently
# labelled as (ACTIVE) will be saved.
SaveGraphicWeb = function(fileName)
{
  
  dev.copy(png, file=fileName, horizontal=F, onefile=F)
  dev.off()

}

# FUNCTION: This function saves tukey graphs as png files. This extra function 
# is necessary due to graph sizes.
SaveGraphicTukeyWeb = function(fileName)
{
  
  dev.copy(png, file=fileName,
  	   horizontal=F, onefile=F)
  dev.off()
  
}

# FUNCTION: This function saves tukey graphs as eps files. This extra function 
# is necessary due to graph sizes.
SaveGraphicTukeyPaper = function(fileName)
{
  
  dev.copy(postscript, file=fileName, height=11, width=5,
           horizontal=F, onefile=F)
  dev.off()
    
}

# FUNCTION: This function is useful if you change this R file and then
# you want the changes to be visible inside of the R environment
reloadCoverageEffectiveness = function()
{

  source("CoverageEffectiveness.R")
  
}

# FUNCTION: This function is responsible for loading all of the
# separate libraries that are needed by the functions in this program.
# This single function can be called if you want to properly load
# all of the libraries for an interactive R session.
librariesCoverageEffectiveness = function()
{

  library(gtools)
  library(rgenoud)
  library(pastecs)

}

########################################################################################################################################################################################################################


DisplayCoverageEffectiveness = function(OrderList,Coverage, Timing, Graph = TRUE)
{

  addvar = FALSE
  
  AllSteps = list()
  
  for(Ordering in OrderList)
	{
	  # the cummulative coverage of the test requirements
	  # note that initially none of the requirements have
	  # been covered by the test suite
	  TotalCoverage = c();

	  # the jump locations inside of the CE step function
	  # note that the first value is always zero 
	  JumpLocations = c();

	  # the between heights for the CE step function
	  # note that the first value is also always zero
	  BetweenHeights = c(0)

	  # this is the running time for "executing" the test suite
	  RunningTime = 0
	  
	  # tried this and I ran into parameter passing problems
	  #sapply(Ordering, GetUniqueCoverage)
	  
	  # apply the function called GetUniqueCoverage to the
	  # Ordering that was provided by the input parameter
	  for ( CurrentTestCase in Ordering )
		{
		  # extract the requirements for the CurrentTestCase
		  CurrentRequirements =
			Coverage$Requirement[Coverage$NameCD == CurrentTestCase]

		  # extract the running time for the CurrentTestCase
		  CurrentTime = Timing$Time[Timing$NameTTD == CurrentTestCase]
		  
		  # determine how many unique requirements this test
		  # case actually covered (probably not needed)
		  MyCoveredRequirements =
			setdiff(CurrentRequirements, TotalCoverage)

		  # update the total coverage as we "run" this next test case
		  TotalCoverage = union(TotalCoverage, CurrentRequirements)

		  # add in the next between height
		  BetweenHeights = c(BetweenHeights, length(TotalCoverage))

		  # add in the next jump location
		  JumpLocations = c(JumpLocations, RunningTime+CurrentTime)

		  # update the running time to reflect the fact that
		  # we "ran" the CurrentTestCase
		  RunningTime = RunningTime + CurrentTime
		} 

	  # create the step function from the JumpLocations and BetweenHeights
	  KappaStepFunction = stepfun(JumpLocations, BetweenHeights)
	 
	 AllSteps = c(AllSteps,list(KappaStepFunction))

	  # the parameter specified that a graph should be produced

	  # make the final coverage effectiveness label
	  CELabel = c("CE = ")
	  CELabelFinal = paste(CELabel, collapse="")

	  # make the final ordering label
	  #FirstOrdering = paste(Ordering, collapse=", ")
	  #OrderingLabel = c(FirstOrdering, " : ", CELabelFinal)
	  #OrderingLabelFinal = paste(OrderingLabel, collapse="")
	   OrderingLabelFinal = "CE"
	  # plot the step function that corresponds to kappa, or the
	  # requirement coverage function
	  
	  color =  colors()[sample(c(153:225))[1]]
	  
	 myplot =  plot.stepfun(KappaStepFunction,
		   xlab = expression(paste("Testing Time  ", (italic(t)))),
                   ylab = expression(paste("Covered Test Requirements  ",(C(T,t)))),
                   xlim=c(0,9),
    		   add=addvar,
		   main = OrderingLabelFinal,
		   do.points=FALSE,
		   col.hor=color,
		   col.vert=color,			
		   cex.main=.975,
		   font.main = 1,
           )
		
		addvar = TRUE
	}
	
	
	#while(TRUE){
		#Sys.sleep(.5)
		#loc = locator(1)
		#identify(AllSteps)
	#}
}



